import math
import os
import sqlite3
from NewsMining.db import dbConnect
import itertools

db = dbConnect.getDBConnection()

def tf_idf(text_word_freq, text_word_count, documents_number, documents_word_count):
    return (text_word_freq / float(text_word_count)) * math.log(documents_number / (float(documents_word_count) + 1), 2)

def dot(x, y):
    return [i * j for i, j in zip(x, y)]

def dot_product(x, y):
    return sum((i * j for i,j in itertools.izip(x, y)))

#-------------------------------------------------------------------------------
# KERNEL
#-------------------------------------------------------------------------------

kernel_words = db.execute("""  select word_id, count(*) as c
                                    from   news_words
                                    where    word_id in (
                                                   select word_id
                                                   from   kernel_words
                                                   )
                                    group by word_id
                                    order by word_id asc""")


kernel_words = dict(kernel_words.fetchall())
kernel_words_length = len(kernel_words)
#-------------------------------------------------------------------------------
# NEWS
#-------------------------------------------------------------------------------
# TOP NEWS
top_news = db.execute("""    select  news_id, class_id
                               from    news
                               where   rank is not null
                               and     length(content) > 300   """)

news = {}

#-------------------------------------------------------------------------------
# DO IT
#-------------------------------------------------------------------------------
for top in top_news:
    news_id = top[0]
    #print news_id
    # GET NEWS WORDS
    news_words = db.execute(""" select   word_id, count
                                from     news_words
                                where    news_id = %i
                                order by word_id asc""" % news_id).fetchall()

    news_words = dict(news_words)
    news_words_length = len(news_words)

    # JOIN NEWS-WORDS WITH KERNEL-WORDS + TF-IDF
    news_kernel_words = [tf_idf(news_words.get(w, 0), news_words_length, kernel_words_length, kernel_words[w]) for w in kernel_words]
    news[str(news_id)] = news_kernel_words

#-------------------------------------------------------------------------------
# CALCULATE DOT_PRODUCT between Examples
#-------------------------------------------------------------------------------
# ONE vs ALL
i = '470'
m = 0
##for j in news:
##    if i != j:
##        d = dot_product(news[i], news[j])
##        if d > 0:
##            m = max(m, d)
##            print "%s - %s == %s" % (i, j, d)

id_row = db.execute("SELECT max(id_row) + 1 FROM kernel_results").fetchone()[0]
id_row = [0, id_row][id_row != None]

# ALL vs ALL
for i in news:
    for j in news:
        if i != j:
            d = dot_product(news[i], news[j])
            if d > 0:
                db.execute("INSERT INTO kernel_results (id_row, news_id_1, news_id_2, similarity, kernel_id) VALUES (%s, %s, %s, %s, %s)" % (id_row, i, j, d, 0))
                id_row += 1

db.commit()